#!/usr/bin/env python
import os, gzip

with_clt = True 
map = {}
for file in os.listdir('.'):
    if file.endswith('.gz'):
        print 'starts with file:', file
        for line in gzip.open(file):
            linebreak = line.split('\t')
            if with_clt or linebreak[-1] == '\n':
                query = linebreak[1]
                if map.has_key(query):
                    map[query] += 1
                else:
                    map[query] = 1

out = open('query_dist', 'w')
for record in sorted(map.items(), lambda x, y: cmp(y[1], x[1])):
    out.write('%s\t%d\n'%record)

out.close()
